var http=require('http');
var https=require('https');
var querystring=require('querystring');
var superagent=require('superagent');
var iconv = require('iconv-lite');
exports.down_load=function(url , callback){
    if(url.indexOf('https')===0){
        //https
        https.get(url,function (res) {
            var data="";
            res.on('data',function(chunk){
                data+=chunk;
            });
            res.on('end',function(){
                callback(data);
            })
        })
    }else if(url.indexOf('http')===0){
        //http
        http.get(url,function (res) {
            var data="";
            res.on('data',function(chunk){
                //如页面编码位GBK请去掉下面注释
                // data+=iconv.decode(chunk, 'gbk')
                data+=chunk;
            });
            res.on('end',function(){
                callback(data);
            })
        })
    }
};
exports.down_page=function(url,callback){
    superagent
        .get(url)
        //设置头部 暂不需要
        .set({})
        .end(function(req,res){
            contexts[contexts.length]={text:"数据爬取完成"};
            // console.log(res.text);
            callback(res.text);
        })
};